/*
 * Comp 479, Fall 2010 Project 1
 *
 * Authors:
 * Boulat Oulmachev #5734266
 * Ivan Khrisanov #9064591
 * Sergiy Samus #4433556
 *
 * Spimi.java
 * Static class that performs SPIMI algorithm
 */

package domain;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.util.HashMap;
import java.util.Map;
import java.util.StringTokenizer;

import model.newmodel.Index;
import model.newmodel.InvalidTermException;
import model.newmodel.Term;
import util.collectionStats.CollectionStatsRegistry;
import util.collectionStats.CollectionStatsRegistry.Stats;
/**
 * @author Ivan Khrisanov, Segriy Samus, Boulat Oulmachev ; Latest reversions: Fabrice Bloncourt, Paul Smelser and Simon Symeonidis
 * @since December 1, 2010
 * @version Modified to read HTML instead of the RUETERS files.
 * Iterates through directories and retrieves all files ending in html or php
 */
public class Spimi {

	private static int docCount=0;
	private static int chkSgmFiles = 0;
    /*
     *  Structure to hold temporary blocks (dictionaries) before they are written on the disk
     */
    public static int execute(String rootPath) throws Exception {

        /*
         *  Getting all the files from the directory
         */
        if (!(new File(rootPath)).exists()) {
            System.out.println("Wrong path: " + rootPath + "\nNothing to index\nPlease check your path");
            System.exit(0);
        }
        
        String[] files = new File(rootPath).list();

        for (String file: files){
        	if ((new File(rootPath+"/"+file)).isDirectory()){
        		Spimi.execute(rootPath+"/"+file);
        	}
        }
        
        int docsCounter = 0;
        int wordsCounter = 0;
        Map<Integer, Integer> docsLengths = new HashMap<Integer, Integer>();

        /*
         *  Selecting sgm files
         */
        for (String file : files) {
            if (file.matches("([^\\s]+(\\.(?i)(html|php|txt))$)")) {
            	chkSgmFiles++;
                Index index = new Index();
                long startTime = System.currentTimeMillis();

                System.out.print("Indexing: " + file);

                /*
                 *  Opening an sgm file
                 */
                BufferedReader reader = new BufferedReader(new FileReader(new File(rootPath + "/" + file)));

                String textLine;
                int docId = docCount++;
                System.out.println(docId);
                int docLength = 0; // in words

                /*
                 *  Reading source file line-by-line
                 */
                while ((textLine = reader.readLine()) != null) 
                {
                	
                    	textLine = textLine.replaceAll("<[^>]*>|&lt;[^>]*>", " ");
                    	textLine = textLine.replaceAll("[\\p{Punct}]", " ");
                    	textLine = textLine.replaceAll("[\\d]", " ");
                    	
                        StringTokenizer st = new StringTokenizer(textLine);
                        while (st.hasMoreTokens()) 
                        {
                            try 
                            {
                                Term term = Term.newInstance(st.nextToken());
                                index.addRecord(term, docId);
                                docLength++;
                                wordsCounter++;
                            } catch (InvalidTermException ignore) {}
                        }
                }

                /*
                 *  Saving index (block) on the disk
                 */
                index.saveTo("C:/Temp/newFiles/tmp-index/" + Integer.toString(chkSgmFiles + 1000000).substring(1) + ".dict");

                /*
                 *  Measuring execution time
                 */
                System.out.println(" (" + (System.currentTimeMillis() - startTime) + " ms.)");
            }
        }

        CollectionStatsRegistry statsRegistry = new CollectionStatsRegistry();
        statsRegistry.setStat(Stats.TOTAL_DOCS, docsCounter);
        statsRegistry.setStat(Stats.AVG_DOC_LENGTH, (double) wordsCounter / docsCounter);
        statsRegistry.setStat(Stats.DOCS_LENGHTS, docsLengths);
        statsRegistry.saveTo(rootPath + "stats.registry");
        
        if (chkSgmFiles == 0) {
            System.out.println("No sgm files found on the path: " + rootPath + "\nNothing to index\nPlease check your path");
            System.exit(0);
        }
        return docCount;
    }
}
